#!/usr/bin/python3
# -*- coding: utf-8 -*-

"""
@version: Python3.6
@author: Silence
@site: 
@software: PyCharm
@file: dy2018Spider.py
@time: 2018/12/5 17:33
"""
import re

from scrapy.http import Request
from scrapy.spiders import CrawlSpider
from scrapy.selector import Selector
from dy2018.dy2018_type import PH_TYPES
from dy2018.items import Dy2018Item


class Spider(CrawlSpider):
    name = 'dy2018Spider'
    host = 'https://www.dy2018.com/'
    start_urls = list(set(PH_TYPES))

    def start_requests(self):
        for ph_type in self.start_urls:
            yield Request(url='https://www.dy2018.com/%s' %ph_type,
                          callback=self.parse_ph_page)

    def parse_ph_page(self, response):
        selector = Selector(response)
        divs = selector.xpath('//div[@class="co_content8"]').extract()
        for div in divs:
            href = re.findall('<a href="(.*?)" class="ulink".*?</a>', div)
            for h in href:
                yield Request(url='https://www.dy2018.com%s' % h, callback=self.parse)
        url_next = selector.xpath('//div[@class="x"]/a')# [-2].xpath('@href').extract()
        for url in url_next:
            if '下一页' == url.xpath('text()').extract_first():
                yield Request(url=self.host + url.xpath('@href').extract_first(), callback=self.parse_ph_page)


    def parse(self, response):
        item = Dy2018Item()
        selector = Selector(response)
        div = selector.xpath('//div[@class="co_area2"]')[0]
        item['video_title'] = div.xpath('//div[@class="title_all"]/h1/text()').extract_first()
        item['video_cover'] = div.re(r'rank">(.*?)</strong>')

        item['video_time'] = div.xpath('//span[@class="updatetime"]/text()').extract_first()
        item['video_type'] = div.xpath('//div[@class="position"]//a/text()').extract()
        download_url = div.xpath('//div[@id="Zoom"]//td//a')
        download = []
        for a in download_url:
            url = a.xpath('@href').extract_first()
            name = a.xpath('text()').extract_first()
            title = a.xpath('@title').extract_first()
            download_item = {'title': title, 'url': url, 'name': name}
            download.append(download_item)
        item['video_download'] = download

        yield item

